import requests, time
import pandas as pd

requests.adapters.DEFAULT_RETRIES = 10
session = requests.session()
session.keep_alive = False
headers = {
    'Accept': 'text/plain, */*; q=0.01',
    'Accept-Encoding': 'gzip, deflate, br',
    'Accept-Language': 'en,zh-CN;q=0.9,zh-TW;q=0.8,zh;q=0.7',
    'Connection': 'keep-alive',
    'Content-Length': '267',
    'Content-Type': 'application/x-www-form-urlencoded',
    'Cookie': 'PHPSESSID=3af6t95emh3bjkgq22m6r7b128; prebid_session=0; adBlockerNewUserDomains=1604913411; StickySession=id.71082202848.548cn.investing.com; _ga=GA1.2.462353728.1604913431; __gads=ID=e2e412b8f1234761:T=1604913440:S=ALNI_MZF3HBQ1gHAjRJLqMn0i1P-jLaoig; logglytrackingsession=f2ba9b18-94de-43e0-894a-7e989802ab4e; r_p_s_n=1; prebid_page=0; SideBlockUser=a%3A2%3A%7Bs%3A10%3A%22stack_size%22%3Ba%3A1%3A%7Bs%3A11%3A%22last_quotes%22%3Bi%3A8%3B%7Ds%3A6%3A%22stacks%22%3Ba%3A1%3A%7Bs%3A11%3A%22last_quotes%22%3Ba%3A4%3A%7Bi%3A0%3Ba%3A3%3A%7Bs%3A7%3A%22pair_ID%22%3Bs%3A3%3A%22169%22%3Bs%3A10%3A%22pair_title%22%3Bs%3A0%3A%22%22%3Bs%3A9%3A%22pair_link%22%3Bs%3A14%3A%22%2Findices%2Fus-30%22%3B%7Di%3A1%3Ba%3A3%3A%7Bs%3A7%3A%22pair_ID%22%3Bs%3A3%3A%22178%22%3Bs%3A10%3A%22pair_title%22%3Bs%3A0%3A%22%22%3Bs%3A9%3A%22pair_link%22%3Bs%3A20%3A%22%2Findices%2Fjapan-ni225%22%3B%7Di%3A2%3Ba%3A3%3A%7Bs%3A7%3A%22pair_ID%22%3Bs%3A3%3A%22179%22%3Bs%3A10%3A%22pair_title%22%3Bs%3A0%3A%22%22%3Bs%3A9%3A%22pair_link%22%3Bs%3A20%3A%22%2Findices%2Fhang-sen-40%22%3B%7Di%3A3%3Ba%3A3%3A%7Bs%3A7%3A%22pair_ID%22%3Bs%3A5%3A%2240820%22%3Bs%3A10%3A%22pair_title%22%3Bs%3A0%3A%22%22%3Bs%3A9%3A%22pair_link%22%3Bs%3A27%3A%22%2Findices%2Fshanghai-composite%22%3B%7D%7D%7D%7D; geoC=CN; comment_notification_219295054=1; UserReactions=true; _gid=GA1.2.1961775226.1605236512; OptanonAlertBoxClosed=2020-11-13T03:04:57.632Z; nyxDorf=NTFiMGAxNHY1azs%2BM35kZzVnM2wxKDA3YGc%3D; OptanonConsent=isIABGlobal=false&datestamp=Fri+Nov+13+2020+11%3A04%3A59+GMT%2B0800+(%E4%B8%AD%E5%9B%BD%E6%A0%87%E5%87%86%E6%97%B6%E9%97%B4)&version=6.7.0&hosts=&landingPath=NotLandingPage&groups=C0001%3A1%2CC0002%3A1%2CC0003%3A1%2CC0004%3A1&AwaitingReconsent=false&geolocation=HK%3BHEA; Hm_lvt_a1e3d50107c2a0e021d734fe76f85914=1604914395,1605236513,1605236531,1605236701; Hm_lpvt_a1e3d50107c2a0e021d734fe76f85914=1605236701; ses_id=ZylmJzM8Ymo%2Bejw6N2YxMmQ8YThibDI1Z2ZiZDU2NSM4LDU7MWYzdWZpO3VlZjgkZWU2MzZjZzA0ZDJoMzI0aGdqZmQzM2JsPj48MzdnMTFkPWE4YmQyZGduYmE1MzU7ODg1ZjE%2BMzJmMTs0ZTk4YmV3Nio2cmd2NGYyYjNyNHNnaGYnM2BiaD45PGQ3MjFiZD1hbmI3MmNnNWJpNWY1LThz',
    'Host': 'cn.investing.com',
    'Origin': 'https://cn.investing.com',
    'Referer': 'https://cn.investing.com/indices/us-30-historical-data',
    'Sec-Fetch-Dest': 'empty',
    'Sec-Fetch-Mode': 'cors',
    'Sec-Fetch-Site': 'same-origin',
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.183 Safari/537.36',
    'X-Requested-With': 'XMLHttpRequest'
}

uri = "https://cn.investing.com/instruments/HistoricalDataAjax"

def _replace_l(x, median_volume):
    if isinstance(x, str):
        if x.strip() == '-':
            x = median_volume
    return x

def get_data(start_date, end_date, invest):
    data = {
        'curr_id': invest['curr_id'],
        'smlID': invest['smlID'],
        'header': invest['header'],
        'st_date': start_date,
        'end_date': end_date,
        'interval_sec': 'Daily',
        'sort_col': 'date',
        'sort_ord': 'DESC',
        'action': 'historical_data'
    }

    response = session.post(url=uri, headers=headers, data=data)
    # pd.set_option('display.max_rows', None)  # 显示所有的行
    # pd.set_option('display.max_columns', None)  # 显示所有的列
    html_data = pd.read_html(response.text)[0]
    df = html_data.rename(columns={'日期': 'open_time', "收盘": "close", "开盘": "open",
                                   "高": "high", "低": "low", "交易量": "volume", "涨跌幅": "change"})
    df['open_time'] = df['open_time'].map(lambda x: cst_to_str(x))
    df['volume'] = df['volume'].map(lambda x: symbol2number(x))
    median_df = df[df['volume'] != "-"]
    median_volume = median_df['volume'].median()

    # 替换 -
    df['volume'] = df['volume'].map(lambda x: _replace_l(x, median_volume))
    df.set_index('open_time', drop=True, inplace=True)
    df.sort_values("open_time", inplace=True)
    # 调换顺序
    df = df[['open','high','low','close','volume', 'change']]
    df['change'] = df['change'].map(lambda x: x.replace("%", ''))
    return df

def cst_to_str(cstTime):
    tempTime = time.strptime(cstTime,'%Y年%m月%d日')
    resTime = time.strftime('%Y-%m-%d',tempTime)
    return resTime

def symbol2number(volume):
    if isinstance(volume, float):
        return volume
    else:
        if 'M' in volume:
            volume = float(volume.replace("M", ""))*1000000
        elif 'B' in volume:
            volume = float(volume.replace("B", ""))*100000000
        elif 'K' in volume:
            volume = float(volume.replace("K", ""))*1000
        return volume
